require 'open-uri'
require 'hpricot'
require 'activerecord'
namespace :crawl_sideffects do
  class SideffectCrawler
    @@websites = []
    def setup
      @output_file = File.open(File.join(File.dirname(__FILE__), "../../vendor/crawl_data/#{self.class.name[0..-8].downcase}.sideffects.txt"), 'w+')
    end
    def do_crawl
    end
    def cleanup
      @output_file.close
    end
    def crawl
      setup
      do_crawl
      cleanup
    end
  end
  
  class MedicinenetCrawler < SideffectCrawler
    
    def setup
      super
      @url_prefix = 'http://www.medicinenet.com/script/main/alphaidx.asp?p='
      @url_suffix = '_sym'
    end
    
    def do_crawl
      'a'.upto('z').each do |initial|
        url = @url_prefix + initial + @url_suffix
        begin
          doc = Hpricot(open(url))
        rescue => e
          puts "error when parsing #{url}: #{e}"
        end
        puts "parsing: #{url}"
        doc.search("//div[@class='AZ_results']/ul/li").each do |item|
          @output_file.puts(item.inner_text.strip)
        end
        sleep 1
      end
    end
  end

  class CureresearchCrawler < SideffectCrawler
    
    def do_crawl
      url = "http://www.cureresearch.com/lists/symptoms.htm"
      begin
        doc = Hpricot(open(url))
      rescue => e
        puts "error when parsing #{url}: #{e}"
      end
      
      doc.search("//div[@class='column-in']//a").each do |item|
        @output_file.puts(item.inner_text.strip) if item.attributes['href'] =~ /\.\.\/sym\/.+\.htm/
      end
    end

  end
  
  class WrongdiagnosisCrawler < SideffectCrawler
    
    def do_crawl
      url = "http://www.wrongdiagnosis.com/lists/symptoms.htm"
      begin
        doc = open(url) do |stream|
          stream.each_line do |line|
            @output_file.puts($1.humanize) if line =~ /\.\.\/sym\/(.+)\.htm/
          end
        end
      rescue => e
        puts "error when parsing #{url}: #{e}"
      end
    end
    
  end
  
  class MedlineplusCrawler < SideffectCrawler
    
    def do_crawl
      ActiveRecord::Base.establish_connection(
        :adapter  => "mysql",
        :host     => "localhost",
        :username => "root",
        :password => "",
        :database => "sideffect_development",
        :port     => 3306
      )
     
      ActiveRecord::Base.connection.select_all('SELECT name FROM sideffects').each do |sd|
        @output_file.puts(sd['name'])
      end
    end
  
  end
  
  
  task :medicinenet do
    MedicinenetCrawler.new.crawl
  end
  
  task :cureresearch do 
    CureresearchCrawler.new.crawl
  end
    
  task :wrongdiagnosis do 
    WrongdiagnosisCrawler.new.crawl
  end
  
  task :medlineplus do 
    MedlineplusCrawler.new.crawl
  end
end
